# -*- coding: utf-8 -*-

# !/usr/bin/env python3
# -*- coding: utf-8 -*-
# Author;Tsukasa

import json
from multiprocessing import Pool
import requests
from requests.exceptions import RequestException
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

ua = UserAgent()
headers1 = {'User-Agent': 'ua.ramdom'}
import re
import pandas as pd
import pymongo
import os

Mongo_Url = 'localhost'
Mongo_DB = 'Lianjia'
t_sub_area = 'sub_area'
t_xiaoqu = 'xiaoqu'
client = pymongo.MongoClient(Mongo_Url)
db = client[Mongo_DB]
base_url = "https://bj.lianjia.com"


def get_area_all_xiaoqu(sub_area_url):
    # print(sub_area_url)
    sub_area_url = sub_area_url[11:]
    print(sub_area_url)
    res = requests.get(base_url + "/xiaoqu" + sub_area_url, 'lxml', headers=headers1)
    total_pages = 1
    if res.status_code == 200:
        soup = BeautifulSoup(res.text, 'lxml')
        page_box = soup.select_one('.house-lst-page-box')
        if page_box is not None and page_box.has_attr('page-data'):
            page_data = page_box['page-data']
            total_pages = page_data[13:].split(',')[0]
    xiaoqu_id_list = []
    # print(total_pages)
    for page in range(1, int(total_pages) + 1):
        res = requests.get(base_url + "/xiaoqu" + sub_area_url + 'pg' + str(page), 'lxml', headers=headers1)
        if res.status_code == 200:
            soup = BeautifulSoup(res.text, 'lxml')
            xiaoqu_id_list += [item['data-housecode'] for item in soup.select('.xiaoquListItem') if
                               item.has_attr('data-housecode')]

    # print(len(xiaoqu_id_list))
    return xiaoqu_id_list


def add_or_update_to_MongoDB(xiaoqu_id):
    print(xiaoqu_id)
    if db[t_xiaoqu].update_one({'小区编号': xiaoqu_id}, {'$set': {"小区编号": xiaoqu_id}}, True):
        print("saved")


if __name__ == '__main__':
    try:
        user_in_city = 'bj'
        print('获取北京所有小区的信息')
        with Pool(os.cpu_count()) as pool:
            for sub_area_info in db[t_sub_area].find().batch_size(50):
                pool.map(add_or_update_to_MongoDB,
                         [xiaoqu_id for xiaoqu_id in get_area_all_xiaoqu(sub_area_info['sub_area_url'])])

    except Exception as ex:
        print(ex)
